{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "import tensorflow as tf\n",
    "\n",
    "import os\n",
    "import pickle\n",
    "import re\n",
    "from tensorflow.python.ops import math_ops"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据预处理为FeatureColumn\n",
    "原始数据文档\n",
    "* https://tianchi.aliyun.com/datalab/dataSet.html?dataId=408\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 处理样本骨架特征\n",
    "\n",
    "### Item Features\t\n",
    "\n",
    "205\tItem ID.\n",
    "\n",
    "206\tCategory ID to which the item belongs to.\n",
    "\n",
    "207\tShop ID to which item belongs to.\n",
    "\n",
    "210\tIntention node ID which the item belongs to.\n",
    "\n",
    "216\tBrand ID of the item.\n",
    "\n",
    "### Combination Features\t\n",
    "508\tThe combination of features with 109_14 and 206.\n",
    "\n",
    "509\tThe combination of features with 110_14 and 207.\n",
    "\n",
    "702\tThe combination of features with 127_14 and 216.\n",
    "\n",
    "853\tThe combination of features with 150_14 and 210.\n",
    "\n",
    "### Context Features\t\n",
    "301\tA categorical expression of position."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练样本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "sample_feature_columns = ['sample_id', 'click', 'buy', 'md5', 'feature_num', 'feature_list']\n",
    "sample_table = pd.read_csv('./ctr_cvr_data/BuyWeight_sample_skeleton_train_sample_2_percent.csv', \n",
    "                             sep=',', header=None, names=sample_feature_columns, engine = 'python')\n",
    "#feature_field_list = ['205','206','207','210','216','508','509','702','853','301']\n",
    "feature_name_list = ['ItemID','CategoryID','ShopID','NodeID','BrandID','Com_CateID',\n",
    "                     'Com_ShopID','Com_BrandID','Com_NodeID','PID']\n",
    "field_id_name = {'205':'ItemID',\n",
    "                 '206':'CategoryID',\n",
    "                 '207':'ShopID',\n",
    "                 '210':'NodeID',\n",
    "                 '216':'BrandID',\n",
    "                 '508':'Com_CateID',\n",
    "                 '509':'Com_ShopID',\n",
    "                 '702':'Com_BrandID',\n",
    "                 '853':'Com_NodeID',\n",
    "                 '301':'PID'}\n",
    "entire_fea_dict = {}\n",
    "for k,v in field_id_name.items():\n",
    "    entire_fea_dict[v] = []\n",
    "for index, row in sample_table.iterrows():\n",
    "    feature_arr = row['feature_list'].split('\\001')\n",
    "    fea_dict = {}\n",
    "    for k,v in field_id_name.items():\n",
    "        fea_dict[k] = []\n",
    "    for fea_kv in feature_arr:\n",
    "        fea_field_id = fea_kv.split('\\002')[0]\n",
    "        fea_id_val = fea_kv.split('\\002')[1]\n",
    "        fea_id = fea_id_val.split('\\003')[0]\n",
    "        fea_val = fea_id_val.split('\\003')[1]\n",
    "        #print(fea_field_id,fea_id,fea_val)\n",
    "        fea_dict[fea_field_id].append(fea_id)\n",
    "    #print(fea_dict)\n",
    "    for k,v in fea_dict.items():\n",
    "        if len(v) == 0:\n",
    "            entire_fea_dict[field_id_name[k]].append('<PAD>')\n",
    "        else:\n",
    "            entire_fea_dict[field_id_name[k]].append('|'.join(v))\n",
    "    if index % 10000 == 0:\n",
    "       print(\"current_index:\",index)\n",
    "\n",
    "#print(entire_fea_dict)    \n",
    "\n",
    "entire_fea_table = pd.DataFrame(data=entire_fea_dict,columns=feature_name_list)\n",
    "\n",
    "#print(sample_table.columns)\n",
    "#print(entire_fea_table.columns)\n",
    "sample_table = sample_table.drop('feature_list',axis=1)\n",
    "\n",
    "sample_table = pd.concat([sample_table, entire_fea_table], axis=1, join_axes=[sample_table.index])\n",
    "\n",
    "sample_table.to_csv('./ctr_cvr_data/BuyWeight_sampled_sample_skeleton_train_sample_feature_column.csv',index=False)\n",
    "print(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 测试集样本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current_index: 0\n",
      "current_index: 10000\n",
      "current_index: 20000\n",
      "current_index: 30000\n",
      "current_index: 40000\n",
      "current_index: 50000\n",
      "current_index: 60000\n",
      "current_index: 70000\n",
      "current_index: 80000\n",
      "current_index: 90000\n",
      "current_index: 100000\n",
      "current_index: 110000\n",
      "current_index: 120000\n",
      "current_index: 130000\n",
      "current_index: 140000\n",
      "current_index: 150000\n",
      "current_index: 160000\n",
      "current_index: 170000\n",
      "current_index: 180000\n",
      "current_index: 190000\n",
      "current_index: 200000\n",
      "current_index: 210000\n",
      "current_index: 220000\n",
      "current_index: 230000\n",
      "current_index: 240000\n",
      "current_index: 250000\n",
      "current_index: 260000\n",
      "current_index: 270000\n",
      "current_index: 280000\n",
      "current_index: 290000\n",
      "current_index: 300000\n",
      "current_index: 310000\n",
      "current_index: 320000\n",
      "current_index: 330000\n",
      "current_index: 340000\n",
      "current_index: 350000\n",
      "current_index: 360000\n",
      "current_index: 370000\n",
      "current_index: 380000\n",
      "current_index: 390000\n",
      "current_index: 400000\n",
      "current_index: 410000\n",
      "current_index: 420000\n",
      "current_index: 430000\n",
      "current_index: 440000\n",
      "current_index: 450000\n",
      "current_index: 460000\n",
      "current_index: 470000\n",
      "current_index: 480000\n",
      "current_index: 490000\n",
      "current_index: 500000\n",
      "current_index: 510000\n",
      "current_index: 520000\n",
      "current_index: 530000\n",
      "current_index: 540000\n",
      "current_index: 550000\n",
      "current_index: 560000\n",
      "current_index: 570000\n",
      "current_index: 580000\n",
      "current_index: 590000\n",
      "current_index: 600000\n",
      "current_index: 610000\n",
      "current_index: 620000\n",
      "current_index: 630000\n",
      "current_index: 640000\n",
      "current_index: 650000\n",
      "current_index: 660000\n",
      "current_index: 670000\n",
      "current_index: 680000\n",
      "current_index: 690000\n",
      "current_index: 700000\n",
      "current_index: 710000\n",
      "current_index: 720000\n",
      "current_index: 730000\n",
      "current_index: 740000\n",
      "current_index: 750000\n",
      "current_index: 760000\n",
      "current_index: 770000\n",
      "current_index: 780000\n",
      "current_index: 790000\n",
      "current_index: 800000\n",
      "current_index: 810000\n",
      "current_index: 820000\n",
      "current_index: 830000\n",
      "current_index: 840000\n",
      "current_index: 850000\n",
      "current_index: 860000\n",
      "current_index: 870000\n",
      "current_index: 880000\n",
      "current_index: 890000\n",
      "current_index: 900000\n",
      "current_index: 910000\n",
      "current_index: 920000\n",
      "current_index: 930000\n",
      "current_index: 940000\n",
      "current_index: 950000\n",
      "current_index: 960000\n",
      "current_index: 970000\n",
      "current_index: 980000\n",
      "current_index: 990000\n",
      "current_index: 1000000\n",
      "current_index: 1010000\n",
      "current_index: 1020000\n",
      "current_index: 1030000\n",
      "current_index: 1040000\n",
      "current_index: 1050000\n",
      "current_index: 1060000\n",
      "current_index: 1070000\n",
      "current_index: 1080000\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "sample_feature_columns = ['sample_id', 'click', 'buy', 'md5', 'feature_num', 'feature_list']\n",
    "sample_table = pd.read_table('./ctr_cvr_data/BuyWeight_sample_skeleton_test_sample_2_percent.csv', \n",
    "                             sep=',', header=None, names=sample_feature_columns, engine = 'python')\n",
    "#feature_field_list = ['205','206','207','210','216','508','509','702','853','301']\n",
    "feature_name_list = ['ItemID','CategoryID','ShopID','NodeID','BrandID','Com_CateID',\n",
    "                     'Com_ShopID','Com_BrandID','Com_NodeID','PID']\n",
    "field_id_name = {'205':'ItemID',\n",
    "                 '206':'CategoryID',\n",
    "                 '207':'ShopID',\n",
    "                 '210':'NodeID',\n",
    "                 '216':'BrandID',\n",
    "                 '508':'Com_CateID',\n",
    "                 '509':'Com_ShopID',\n",
    "                 '702':'Com_BrandID',\n",
    "                 '853':'Com_NodeID',\n",
    "                 '301':'PID'}\n",
    "entire_fea_dict = {}\n",
    "for k,v in field_id_name.items():\n",
    "    entire_fea_dict[v] = []\n",
    "for index, row in sample_table.iterrows():\n",
    "    feature_arr = row['feature_list'].split('\\001')\n",
    "    fea_dict = {}\n",
    "    for k,v in field_id_name.items():\n",
    "        fea_dict[k] = []\n",
    "    for fea_kv in feature_arr:\n",
    "        fea_field_id = fea_kv.split('\\002')[0]\n",
    "        fea_id_val = fea_kv.split('\\002')[1]\n",
    "        fea_id = fea_id_val.split('\\003')[0]\n",
    "        fea_val = fea_id_val.split('\\003')[1]\n",
    "        #print(fea_field_id,fea_id,fea_val)\n",
    "        fea_dict[fea_field_id].append(fea_id)\n",
    "    #print(fea_dict)\n",
    "    for k,v in fea_dict.items():\n",
    "        if len(v) == 0:\n",
    "            entire_fea_dict[field_id_name[k]].append('<PAD>')\n",
    "        else:\n",
    "            entire_fea_dict[field_id_name[k]].append('|'.join(v))\n",
    "    if index % 10000 == 0:\n",
    "       print(\"current_index:\",index)\n",
    "\n",
    "#print(entire_fea_dict)    \n",
    "\n",
    "entire_fea_table = pd.DataFrame(data=entire_fea_dict,columns=feature_name_list)\n",
    "\n",
    "#print(sample_table.columns)\n",
    "#print(entire_fea_table.columns)\n",
    "sample_table = sample_table.drop('feature_list',axis=1)\n",
    "\n",
    "sample_table = pd.concat([sample_table, entire_fea_table], axis=1, join_axes=[sample_table.index])\n",
    "\n",
    "sample_table.to_csv('./ctr_cvr_data/BuyWeight_sampled_sample_skeleton_test_sample_feature_column.csv',index=False)\n",
    "print(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 处理Common 用户特征\n",
    "### User Features\t\n",
    "101\tUser ID.\n",
    "\n",
    "109_14\tUser historical behaviors of category ID and count*.\n",
    "\n",
    "110_14\tUser historical behaviors of shop ID and count*.\n",
    "\n",
    "127_14\tUser historical behaviors of brand ID and count*.\n",
    "\n",
    "150_14\tUser historical behaviors of intention node ID and count*.\n",
    "\n",
    "121\tCategorical ID of User Profile.\n",
    "\n",
    "122\tCategorical group ID of User Profile.\n",
    "\n",
    "124\tUsers Gender ID.\n",
    "\n",
    "125\tUsers Age ID.\n",
    "\n",
    "126\tUsers Consumption Level Type I.\n",
    "\n",
    "127\tUsers Consumption Level Type II.\n",
    "\n",
    "128\tUsers Occupation: whether or not to work.\n",
    "\n",
    "129\tUsers Geography Informations."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练集common feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "black_list = set(['109_14','110_14','127_14','150_14'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current_index: 0\n",
      "current_index: 1000\n",
      "current_index: 2000\n",
      "current_index: 3000\n",
      "current_index: 4000\n",
      "current_index: 5000\n",
      "current_index: 6000\n",
      "current_index: 7000\n",
      "current_index: 8000\n",
      "current_index: 9000\n",
      "current_index: 10000\n",
      "current_index: 11000\n",
      "current_index: 12000\n",
      "current_index: 13000\n",
      "current_index: 14000\n",
      "current_index: 15000\n",
      "current_index: 16000\n",
      "current_index: 17000\n",
      "current_index: 18000\n",
      "current_index: 19000\n",
      "current_index: 20000\n",
      "current_index: 21000\n",
      "current_index: 22000\n",
      "current_index: 23000\n",
      "current_index: 24000\n",
      "current_index: 25000\n",
      "current_index: 26000\n",
      "current_index: 27000\n",
      "current_index: 28000\n",
      "current_index: 29000\n",
      "current_index: 30000\n",
      "current_index: 31000\n",
      "current_index: 32000\n",
      "current_index: 33000\n",
      "current_index: 34000\n",
      "current_index: 35000\n",
      "current_index: 36000\n",
      "current_index: 37000\n",
      "current_index: 38000\n",
      "current_index: 39000\n",
      "current_index: 40000\n",
      "current_index: 41000\n",
      "current_index: 42000\n",
      "current_index: 43000\n",
      "current_index: 44000\n",
      "current_index: 45000\n",
      "current_index: 46000\n",
      "current_index: 47000\n",
      "current_index: 48000\n",
      "current_index: 49000\n",
      "current_index: 50000\n",
      "current_index: 51000\n",
      "current_index: 52000\n",
      "current_index: 53000\n",
      "current_index: 54000\n",
      "current_index: 55000\n",
      "current_index: 56000\n",
      "current_index: 57000\n",
      "current_index: 58000\n",
      "current_index: 59000\n",
      "current_index: 60000\n",
      "current_index: 61000\n",
      "current_index: 62000\n",
      "current_index: 63000\n",
      "current_index: 64000\n",
      "current_index: 65000\n",
      "current_index: 66000\n",
      "current_index: 67000\n",
      "current_index: 68000\n",
      "current_index: 69000\n",
      "current_index: 70000\n",
      "current_index: 71000\n",
      "current_index: 72000\n",
      "current_index: 73000\n",
      "current_index: 74000\n",
      "current_index: 75000\n",
      "current_index: 76000\n",
      "current_index: 77000\n",
      "current_index: 78000\n",
      "current_index: 79000\n",
      "current_index: 80000\n",
      "current_index: 81000\n",
      "current_index: 82000\n",
      "current_index: 83000\n",
      "current_index: 84000\n",
      "current_index: 85000\n",
      "current_index: 86000\n",
      "current_index: 87000\n",
      "current_index: 88000\n",
      "current_index: 89000\n",
      "current_index: 90000\n",
      "current_index: 91000\n",
      "current_index: 92000\n",
      "current_index: 93000\n",
      "current_index: 94000\n",
      "current_index: 95000\n",
      "current_index: 96000\n",
      "current_index: 97000\n",
      "current_index: 98000\n",
      "current_index: 99000\n",
      "current_index: 100000\n",
      "current_index: 101000\n",
      "current_index: 102000\n",
      "current_index: 103000\n",
      "current_index: 104000\n",
      "current_index: 105000\n",
      "current_index: 106000\n",
      "current_index: 107000\n",
      "current_index: 108000\n",
      "current_index: 109000\n",
      "current_index: 110000\n",
      "current_index: 111000\n",
      "current_index: 112000\n",
      "current_index: 113000\n",
      "current_index: 114000\n",
      "current_index: 115000\n",
      "current_index: 116000\n",
      "current_index: 117000\n",
      "current_index: 118000\n",
      "current_index: 119000\n",
      "current_index: 120000\n",
      "current_index: 121000\n",
      "current_index: 122000\n",
      "current_index: 123000\n",
      "current_index: 124000\n",
      "current_index: 125000\n",
      "current_index: 126000\n",
      "current_index: 127000\n",
      "current_index: 128000\n",
      "current_index: 129000\n",
      "current_index: 130000\n",
      "current_index: 131000\n",
      "current_index: 132000\n",
      "current_index: 133000\n",
      "current_index: 134000\n",
      "current_index: 135000\n",
      "current_index: 136000\n",
      "current_index: 137000\n",
      "current_index: 138000\n",
      "current_index: 139000\n",
      "current_index: 140000\n",
      "current_index: 141000\n",
      "current_index: 142000\n",
      "current_index: 143000\n",
      "current_index: 144000\n",
      "current_index: 145000\n",
      "current_index: 146000\n",
      "current_index: 147000\n",
      "current_index: 148000\n",
      "current_index: 149000\n",
      "current_index: 150000\n",
      "current_index: 151000\n",
      "current_index: 152000\n",
      "current_index: 153000\n",
      "current_index: 154000\n",
      "current_index: 155000\n",
      "current_index: 156000\n",
      "current_index: 157000\n",
      "current_index: 158000\n",
      "current_index: 159000\n",
      "current_index: 160000\n",
      "current_index: 161000\n",
      "current_index: 162000\n",
      "current_index: 163000\n",
      "current_index: 164000\n",
      "current_index: 165000\n",
      "current_index: 166000\n",
      "current_index: 167000\n",
      "current_index: 168000\n",
      "current_index: 169000\n",
      "current_index: 170000\n",
      "current_index: 171000\n",
      "current_index: 172000\n",
      "current_index: 173000\n",
      "current_index: 174000\n",
      "current_index: 175000\n",
      "current_index: 176000\n",
      "current_index: 177000\n",
      "current_index: 178000\n",
      "current_index: 179000\n",
      "current_index: 180000\n",
      "current_index: 181000\n",
      "current_index: 182000\n",
      "current_index: 183000\n",
      "current_index: 184000\n",
      "current_index: 185000\n",
      "current_index: 186000\n",
      "current_index: 187000\n",
      "current_index: 188000\n",
      "current_index: 189000\n",
      "current_index: 190000\n",
      "current_index: 191000\n",
      "current_index: 192000\n",
      "current_index: 193000\n",
      "current_index: 194000\n",
      "current_index: 195000\n",
      "current_index: 196000\n",
      "current_index: 197000\n",
      "current_index: 198000\n",
      "current_index: 199000\n",
      "current_index: 200000\n",
      "current_index: 201000\n",
      "current_index: 202000\n",
      "current_index: 203000\n",
      "current_index: 204000\n",
      "current_index: 205000\n",
      "current_index: 206000\n",
      "current_index: 207000\n",
      "current_index: 208000\n",
      "current_index: 209000\n",
      "current_index: 210000\n",
      "current_index: 211000\n",
      "current_index: 212000\n",
      "current_index: 213000\n",
      "current_index: 214000\n",
      "current_index: 215000\n",
      "current_index: 216000\n",
      "current_index: 217000\n",
      "current_index: 218000\n",
      "current_index: 219000\n",
      "current_index: 220000\n",
      "current_index: 221000\n",
      "current_index: 222000\n",
      "current_index: 223000\n",
      "current_index: 224000\n",
      "current_index: 225000\n",
      "current_index: 226000\n",
      "current_index: 227000\n",
      "current_index: 228000\n",
      "current_index: 229000\n",
      "current_index: 230000\n",
      "current_index: 231000\n",
      "current_index: 232000\n",
      "current_index: 233000\n",
      "current_index: 234000\n",
      "current_index: 235000\n",
      "current_index: 236000\n",
      "current_index: 237000\n",
      "current_index: 238000\n",
      "current_index: 239000\n",
      "current_index: 240000\n",
      "current_index: 241000\n",
      "current_index: 242000\n",
      "current_index: 243000\n",
      "current_index: 244000\n",
      "current_index: 245000\n",
      "current_index: 246000\n",
      "current_index: 247000\n",
      "current_index: 248000\n",
      "current_index: 249000\n",
      "current_index: 250000\n",
      "current_index: 251000\n",
      "current_index: 252000\n",
      "current_index: 253000\n",
      "current_index: 254000\n",
      "current_index: 255000\n",
      "current_index: 256000\n",
      "current_index: 257000\n",
      "current_index: 258000\n",
      "current_index: 259000\n",
      "current_index: 260000\n",
      "current_index: 261000\n",
      "current_index: 262000\n",
      "current_index: 263000\n",
      "current_index: 264000\n",
      "current_index: 265000\n",
      "current_index: 266000\n",
      "current_index: 267000\n",
      "current_index: 268000\n",
      "current_index: 269000\n",
      "current_index: 270000\n",
      "current_index: 271000\n",
      "current_index: 272000\n",
      "current_index: 273000\n",
      "current_index: 274000\n",
      "current_index: 275000\n",
      "current_index: 276000\n",
      "current_index: 277000\n",
      "current_index: 278000\n",
      "current_index: 279000\n",
      "current_index: 280000\n",
      "current_index: 281000\n",
      "current_index: 282000\n",
      "current_index: 283000\n",
      "current_index: 284000\n",
      "current_index: 285000\n",
      "current_index: 286000\n",
      "current_index: 287000\n",
      "current_index: 288000\n",
      "current_index: 289000\n",
      "current_index: 290000\n",
      "current_index: 291000\n",
      "current_index: 292000\n",
      "current_index: 293000\n",
      "current_index: 294000\n",
      "current_index: 295000\n",
      "current_index: 296000\n",
      "current_index: 297000\n",
      "current_index: 298000\n",
      "current_index: 299000\n",
      "current_index: 300000\n",
      "current_index: 301000\n",
      "current_index: 302000\n",
      "current_index: 303000\n",
      "current_index: 304000\n",
      "current_index: 305000\n",
      "current_index: 306000\n",
      "current_index: 307000\n",
      "current_index: 308000\n",
      "current_index: 309000\n",
      "current_index: 310000\n",
      "current_index: 311000\n",
      "current_index: 312000\n",
      "current_index: 313000\n",
      "current_index: 314000\n",
      "current_index: 315000\n",
      "current_index: 316000\n",
      "current_index: 317000\n",
      "current_index: 318000\n",
      "current_index: 319000\n",
      "current_index: 320000\n",
      "current_index: 321000\n",
      "current_index: 322000\n",
      "current_index: 323000\n",
      "current_index: 324000\n",
      "current_index: 325000\n",
      "current_index: 326000\n",
      "current_index: 327000\n",
      "current_index: 328000\n",
      "current_index: 329000\n",
      "current_index: 330000\n",
      "current_index: 331000\n",
      "current_index: 332000\n",
      "current_index: 333000\n",
      "current_index: 334000\n",
      "current_index: 335000\n",
      "current_index: 336000\n",
      "current_index: 337000\n",
      "current_index: 338000\n",
      "current_index: 339000\n",
      "current_index: 340000\n",
      "current_index: 341000\n",
      "current_index: 342000\n",
      "current_index: 343000\n",
      "current_index: 344000\n",
      "current_index: 345000\n",
      "current_index: 346000\n",
      "current_index: 347000\n",
      "current_index: 348000\n",
      "current_index: 349000\n",
      "current_index: 350000\n",
      "current_index: 351000\n",
      "current_index: 352000\n",
      "current_index: 353000\n",
      "current_index: 354000\n",
      "current_index: 355000\n",
      "current_index: 356000\n",
      "current_index: 357000\n",
      "current_index: 358000\n",
      "current_index: 359000\n",
      "current_index: 360000\n",
      "current_index: 361000\n",
      "current_index: 362000\n",
      "current_index: 363000\n",
      "current_index: 364000\n",
      "current_index: 365000\n",
      "current_index: 366000\n",
      "current_index: 367000\n",
      "current_index: 368000\n",
      "current_index: 369000\n",
      "current_index: 370000\n",
      "current_index: 371000\n",
      "current_index: 372000\n",
      "current_index: 373000\n",
      "current_index: 374000\n",
      "current_index: 375000\n",
      "current_index: 376000\n",
      "current_index: 377000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current_index: 378000\n",
      "current_index: 379000\n",
      "current_index: 380000\n",
      "current_index: 381000\n",
      "current_index: 382000\n",
      "current_index: 383000\n",
      "current_index: 384000\n",
      "current_index: 385000\n",
      "current_index: 386000\n",
      "current_index: 387000\n",
      "current_index: 388000\n",
      "current_index: 389000\n",
      "current_index: 390000\n",
      "current_index: 391000\n",
      "current_index: 392000\n",
      "current_index: 393000\n",
      "current_index: 394000\n",
      "current_index: 395000\n",
      "current_index: 396000\n",
      "current_index: 397000\n",
      "current_index: 398000\n",
      "current_index: 399000\n",
      "current_index: 400000\n",
      "current_index: 401000\n",
      "current_index: 402000\n",
      "current_index: 403000\n",
      "current_index: 404000\n",
      "current_index: 405000\n",
      "current_index: 406000\n",
      "current_index: 407000\n",
      "current_index: 408000\n",
      "current_index: 409000\n",
      "current_index: 410000\n",
      "current_index: 411000\n",
      "current_index: 412000\n",
      "current_index: 413000\n",
      "current_index: 414000\n",
      "current_index: 415000\n",
      "current_index: 416000\n",
      "current_index: 417000\n",
      "current_index: 418000\n",
      "current_index: 419000\n",
      "current_index: 420000\n",
      "current_index: 421000\n",
      "current_index: 422000\n",
      "current_index: 423000\n",
      "current_index: 424000\n",
      "current_index: 425000\n",
      "current_index: 426000\n",
      "current_index: 427000\n",
      "current_index: 428000\n",
      "current_index: 429000\n",
      "current_index: 430000\n",
      "current_index: 431000\n",
      "current_index: 432000\n",
      "current_index: 433000\n",
      "current_index: 434000\n",
      "current_index: 435000\n",
      "current_index: 436000\n",
      "current_index: 437000\n",
      "current_index: 438000\n",
      "current_index: 439000\n",
      "current_index: 440000\n",
      "current_index: 441000\n",
      "current_index: 442000\n",
      "current_index: 443000\n",
      "current_index: 444000\n",
      "current_index: 445000\n",
      "current_index: 446000\n",
      "current_index: 447000\n",
      "current_index: 448000\n",
      "current_index: 449000\n",
      "current_index: 450000\n",
      "current_index: 451000\n",
      "current_index: 452000\n",
      "current_index: 453000\n",
      "current_index: 454000\n",
      "current_index: 455000\n",
      "current_index: 456000\n",
      "current_index: 457000\n",
      "current_index: 458000\n",
      "current_index: 459000\n",
      "current_index: 460000\n",
      "current_index: 461000\n",
      "current_index: 462000\n",
      "current_index: 463000\n",
      "current_index: 464000\n",
      "current_index: 465000\n",
      "current_index: 466000\n",
      "current_index: 467000\n",
      "(467621, 13)\n",
      "(467621, 3)\n",
      "(467621, 15)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "common_table_columns = ['md5', 'feature_num', 'feature_list']\n",
    "common_table = pd.read_table('./ctr_cvr_data/BuyWeight_common_features_skeleton_train_sample_2_percent.csv', \n",
    "                                sep=',', header=None, names=common_table_columns, engine = 'python')\n",
    "feature_name_list = ['UserID', 'User_CateIDs', 'User_ShopIDs', 'User_BrandIDs', 'User_NodeIDs', 'User_Cluster', \n",
    "                     'User_ClusterID', 'User_Gender', 'User_Age', 'User_Level1', 'User_Level2', \n",
    "                     'User_Occupation', 'User_Geo']\n",
    "field_id_name = {'101':'UserID',\n",
    "                 '109_14':'User_CateIDs',\n",
    "                 '110_14':'User_ShopIDs',\n",
    "                 '127_14':'User_BrandIDs',\n",
    "                 '150_14':'User_NodeIDs',\n",
    "                 '121':'User_Cluster',\n",
    "                 '122':'User_ClusterID',\n",
    "                 '124':'User_Gender',\n",
    "                 '125':'User_Age',\n",
    "                 '126':'User_Level1',\n",
    "                 '127':'User_Level2',\n",
    "                 '128':'User_Occupation',\n",
    "                 '129':'User_Geo'}\n",
    "\n",
    "#black_list = set(['109_14','110_14','127_14','150_14'])\n",
    "black_list = set(['110_14','150_14'])\n",
    "entire_fea_dict = {}\n",
    "for k,v in field_id_name.items():\n",
    "    entire_fea_dict[v] = []\n",
    "for index, row in common_table.iterrows():\n",
    "    feature_arr = row['feature_list'].split('\\001')\n",
    "    fea_dict = {}\n",
    "    for k,v in field_id_name.items():\n",
    "        fea_dict[k] = []\n",
    "    for fea_kv in feature_arr:\n",
    "        fea_field_id = fea_kv.split('\\002')[0]\n",
    "        fea_id_val = fea_kv.split('\\002')[1]\n",
    "        fea_id = fea_id_val.split('\\003')[0]\n",
    "        fea_val = fea_id_val.split('\\003')[1]\n",
    "        #print(fea_field_id,fea_id,fea_val)\n",
    "        if fea_field_id in black_list:\n",
    "            continue\n",
    "        # Multi-Hot IDs类特征保留前100个ID\n",
    "        if len(fea_dict[fea_field_id]) < 100:\n",
    "            fea_dict[fea_field_id].append(fea_id)\n",
    "    #print(fea_dict)\n",
    "    for k,v in fea_dict.items():\n",
    "        if len(v) == 0:\n",
    "            entire_fea_dict[field_id_name[k]].append('<PAD>')\n",
    "        else:\n",
    "            entire_fea_dict[field_id_name[k]].append('|'.join(v))\n",
    "    if index % 1000 == 0:\n",
    "       print(\"current_index:\",index)\n",
    "#print(entire_fea_dict)    \n",
    "\n",
    "entire_fea_table = pd.DataFrame(data=entire_fea_dict, columns=feature_name_list)\n",
    "print(entire_fea_table.shape)\n",
    "print(common_table.shape)\n",
    "common_table = common_table.drop('feature_list',axis=1)\n",
    "\n",
    "common_table = pd.concat([common_table, entire_fea_table], axis=1, join_axes=[common_table.index])\n",
    "\n",
    "common_table.to_csv('./ctr_cvr_data/BuyWeight_sampled_common_features_skeleton_train_sample_feature_column.csv',index=False)\n",
    "print(common_table.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>md5</th>\n",
       "      <th>feature_num</th>\n",
       "      <th>UserID</th>\n",
       "      <th>User_CateIDs</th>\n",
       "      <th>User_ShopIDs</th>\n",
       "      <th>User_BrandIDs</th>\n",
       "      <th>User_NodeIDs</th>\n",
       "      <th>User_Cluster</th>\n",
       "      <th>User_ClusterID</th>\n",
       "      <th>User_Gender</th>\n",
       "      <th>User_Age</th>\n",
       "      <th>User_Level1</th>\n",
       "      <th>User_Level2</th>\n",
       "      <th>User_Occupation</th>\n",
       "      <th>User_Geo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>84dceed2e3a667f8</td>\n",
       "      <td>343</td>\n",
       "      <td>31319</td>\n",
       "      <td>450877|447414|446442|450989|451636|449082|4572...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3781041|3850935|3850235|3638768|3858194|359279...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3438687</td>\n",
       "      <td>3438762</td>\n",
       "      <td>3438769</td>\n",
       "      <td>3438774</td>\n",
       "      <td>3438779</td>\n",
       "      <td>3438782</td>\n",
       "      <td>3864885</td>\n",
       "      <td>3864888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000350f0c2121e7</td>\n",
       "      <td>811</td>\n",
       "      <td>392326</td>\n",
       "      <td>447553|445995|450247|449070|450980|445135|4454...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3716224|3514627|3772871|3543283|3728186|371080...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3438725</td>\n",
       "      <td>3438760</td>\n",
       "      <td>3438769</td>\n",
       "      <td>3438772</td>\n",
       "      <td>3438778</td>\n",
       "      <td>3438782</td>\n",
       "      <td>3864885</td>\n",
       "      <td>3864888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000091a89d1867ab</td>\n",
       "      <td>7</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3438658</td>\n",
       "      <td>3438761</td>\n",
       "      <td>3438769</td>\n",
       "      <td>3438773</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3438781</td>\n",
       "      <td>3864885</td>\n",
       "      <td>3864889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0001fa8246be0940</td>\n",
       "      <td>374</td>\n",
       "      <td>407969</td>\n",
       "      <td>451311|450954|450462|451530|451099|450656|4490...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3504052|3507496|3622158|3630324|3566530|352097...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3438737</td>\n",
       "      <td>3438757</td>\n",
       "      <td>3438768</td>\n",
       "      <td>3438774</td>\n",
       "      <td>3438778</td>\n",
       "      <td>3438782</td>\n",
       "      <td>3864885</td>\n",
       "      <td>3864888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000260b23f85aadb</td>\n",
       "      <td>266</td>\n",
       "      <td>168295</td>\n",
       "      <td>450837|451033|450838|449949|455349|455827|4553...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3627914|3760360|3763560|3496527|3689932|384471...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3438705</td>\n",
       "      <td>3438765</td>\n",
       "      <td>3438768</td>\n",
       "      <td>3438771</td>\n",
       "      <td>3438777</td>\n",
       "      <td>3438782</td>\n",
       "      <td>3864886</td>\n",
       "      <td>3864888</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                md5  feature_num  UserID  \\\n",
       "0  84dceed2e3a667f8          343   31319   \n",
       "1  0000350f0c2121e7          811  392326   \n",
       "2  000091a89d1867ab            7   <PAD>   \n",
       "3  0001fa8246be0940          374  407969   \n",
       "4  000260b23f85aadb          266  168295   \n",
       "\n",
       "                                        User_CateIDs User_ShopIDs  \\\n",
       "0  450877|447414|446442|450989|451636|449082|4572...        <PAD>   \n",
       "1  447553|445995|450247|449070|450980|445135|4454...        <PAD>   \n",
       "2                                              <PAD>        <PAD>   \n",
       "3  451311|450954|450462|451530|451099|450656|4490...        <PAD>   \n",
       "4  450837|451033|450838|449949|455349|455827|4553...        <PAD>   \n",
       "\n",
       "                                       User_BrandIDs User_NodeIDs  \\\n",
       "0  3781041|3850935|3850235|3638768|3858194|359279...        <PAD>   \n",
       "1  3716224|3514627|3772871|3543283|3728186|371080...        <PAD>   \n",
       "2                                              <PAD>        <PAD>   \n",
       "3  3504052|3507496|3622158|3630324|3566530|352097...        <PAD>   \n",
       "4  3627914|3760360|3763560|3496527|3689932|384471...        <PAD>   \n",
       "\n",
       "  User_Cluster User_ClusterID User_Gender User_Age User_Level1 User_Level2  \\\n",
       "0      3438687        3438762     3438769  3438774     3438779     3438782   \n",
       "1      3438725        3438760     3438769  3438772     3438778     3438782   \n",
       "2      3438658        3438761     3438769  3438773       <PAD>     3438781   \n",
       "3      3438737        3438757     3438768  3438774     3438778     3438782   \n",
       "4      3438705        3438765     3438768  3438771     3438777     3438782   \n",
       "\n",
       "  User_Occupation User_Geo  \n",
       "0         3864885  3864888  \n",
       "1         3864885  3864888  \n",
       "2         3864885  3864889  \n",
       "3         3864885  3864888  \n",
       "4         3864886  3864888  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "common_table.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 测试集common feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current_index: 0\n",
      "current_index: 1000\n",
      "current_index: 2000\n",
      "current_index: 3000\n",
      "current_index: 4000\n",
      "current_index: 5000\n",
      "current_index: 6000\n",
      "current_index: 7000\n",
      "current_index: 8000\n",
      "current_index: 9000\n",
      "current_index: 10000\n",
      "current_index: 11000\n",
      "current_index: 12000\n",
      "current_index: 13000\n",
      "current_index: 14000\n",
      "current_index: 15000\n",
      "current_index: 16000\n",
      "current_index: 17000\n",
      "current_index: 18000\n",
      "current_index: 19000\n",
      "current_index: 20000\n",
      "current_index: 21000\n",
      "current_index: 22000\n",
      "current_index: 23000\n",
      "current_index: 24000\n",
      "current_index: 25000\n",
      "current_index: 26000\n",
      "current_index: 27000\n",
      "current_index: 28000\n",
      "current_index: 29000\n",
      "current_index: 30000\n",
      "current_index: 31000\n",
      "current_index: 32000\n",
      "current_index: 33000\n",
      "current_index: 34000\n",
      "current_index: 35000\n",
      "current_index: 36000\n",
      "current_index: 37000\n",
      "current_index: 38000\n",
      "current_index: 39000\n",
      "current_index: 40000\n",
      "current_index: 41000\n",
      "current_index: 42000\n",
      "current_index: 43000\n",
      "current_index: 44000\n",
      "current_index: 45000\n",
      "current_index: 46000\n",
      "current_index: 47000\n",
      "current_index: 48000\n",
      "current_index: 49000\n",
      "current_index: 50000\n",
      "current_index: 51000\n",
      "current_index: 52000\n",
      "current_index: 53000\n",
      "current_index: 54000\n",
      "current_index: 55000\n",
      "current_index: 56000\n",
      "current_index: 57000\n",
      "current_index: 58000\n",
      "current_index: 59000\n",
      "current_index: 60000\n",
      "current_index: 61000\n",
      "current_index: 62000\n",
      "current_index: 63000\n",
      "current_index: 64000\n",
      "current_index: 65000\n",
      "current_index: 66000\n",
      "current_index: 67000\n",
      "current_index: 68000\n",
      "current_index: 69000\n",
      "current_index: 70000\n",
      "current_index: 71000\n",
      "current_index: 72000\n",
      "current_index: 73000\n",
      "current_index: 74000\n",
      "current_index: 75000\n",
      "current_index: 76000\n",
      "current_index: 77000\n",
      "current_index: 78000\n",
      "current_index: 79000\n",
      "current_index: 80000\n",
      "current_index: 81000\n",
      "current_index: 82000\n",
      "current_index: 83000\n",
      "current_index: 84000\n",
      "current_index: 85000\n",
      "current_index: 86000\n",
      "current_index: 87000\n",
      "current_index: 88000\n",
      "current_index: 89000\n",
      "current_index: 90000\n",
      "current_index: 91000\n",
      "current_index: 92000\n",
      "current_index: 93000\n",
      "current_index: 94000\n",
      "current_index: 95000\n",
      "current_index: 96000\n",
      "current_index: 97000\n",
      "current_index: 98000\n",
      "current_index: 99000\n",
      "current_index: 100000\n",
      "current_index: 101000\n",
      "current_index: 102000\n",
      "current_index: 103000\n",
      "current_index: 104000\n",
      "current_index: 105000\n",
      "current_index: 106000\n",
      "current_index: 107000\n",
      "current_index: 108000\n",
      "current_index: 109000\n",
      "current_index: 110000\n",
      "current_index: 111000\n",
      "current_index: 112000\n",
      "current_index: 113000\n",
      "current_index: 114000\n",
      "current_index: 115000\n",
      "current_index: 116000\n",
      "current_index: 117000\n",
      "current_index: 118000\n",
      "current_index: 119000\n",
      "current_index: 120000\n",
      "current_index: 121000\n",
      "current_index: 122000\n",
      "current_index: 123000\n",
      "current_index: 124000\n",
      "current_index: 125000\n",
      "current_index: 126000\n",
      "current_index: 127000\n",
      "current_index: 128000\n",
      "current_index: 129000\n",
      "current_index: 130000\n",
      "current_index: 131000\n",
      "current_index: 132000\n",
      "current_index: 133000\n",
      "current_index: 134000\n",
      "current_index: 135000\n",
      "current_index: 136000\n",
      "current_index: 137000\n",
      "current_index: 138000\n",
      "current_index: 139000\n",
      "current_index: 140000\n",
      "current_index: 141000\n",
      "current_index: 142000\n",
      "current_index: 143000\n",
      "current_index: 144000\n",
      "current_index: 145000\n",
      "current_index: 146000\n",
      "current_index: 147000\n",
      "current_index: 148000\n",
      "current_index: 149000\n",
      "current_index: 150000\n",
      "current_index: 151000\n",
      "current_index: 152000\n",
      "current_index: 153000\n",
      "current_index: 154000\n",
      "current_index: 155000\n",
      "current_index: 156000\n",
      "current_index: 157000\n",
      "current_index: 158000\n",
      "current_index: 159000\n",
      "current_index: 160000\n",
      "current_index: 161000\n",
      "current_index: 162000\n",
      "current_index: 163000\n",
      "current_index: 164000\n",
      "current_index: 165000\n",
      "current_index: 166000\n",
      "current_index: 167000\n",
      "current_index: 168000\n",
      "current_index: 169000\n",
      "current_index: 170000\n",
      "current_index: 171000\n",
      "current_index: 172000\n",
      "current_index: 173000\n",
      "current_index: 174000\n",
      "current_index: 175000\n",
      "current_index: 176000\n",
      "current_index: 177000\n",
      "current_index: 178000\n",
      "current_index: 179000\n",
      "current_index: 180000\n",
      "current_index: 181000\n",
      "current_index: 182000\n",
      "current_index: 183000\n",
      "current_index: 184000\n",
      "current_index: 185000\n",
      "current_index: 186000\n",
      "current_index: 187000\n",
      "current_index: 188000\n",
      "current_index: 189000\n",
      "current_index: 190000\n",
      "current_index: 191000\n",
      "current_index: 192000\n",
      "current_index: 193000\n",
      "current_index: 194000\n",
      "current_index: 195000\n",
      "current_index: 196000\n",
      "current_index: 197000\n",
      "current_index: 198000\n",
      "current_index: 199000\n",
      "current_index: 200000\n",
      "current_index: 201000\n",
      "current_index: 202000\n",
      "current_index: 203000\n",
      "current_index: 204000\n",
      "current_index: 205000\n",
      "current_index: 206000\n",
      "current_index: 207000\n",
      "current_index: 208000\n",
      "current_index: 209000\n",
      "current_index: 210000\n",
      "current_index: 211000\n",
      "current_index: 212000\n",
      "current_index: 213000\n",
      "current_index: 214000\n",
      "current_index: 215000\n",
      "current_index: 216000\n",
      "current_index: 217000\n",
      "current_index: 218000\n",
      "current_index: 219000\n",
      "current_index: 220000\n",
      "current_index: 221000\n",
      "current_index: 222000\n",
      "current_index: 223000\n",
      "current_index: 224000\n",
      "current_index: 225000\n",
      "current_index: 226000\n",
      "current_index: 227000\n",
      "current_index: 228000\n",
      "current_index: 229000\n",
      "current_index: 230000\n",
      "current_index: 231000\n",
      "current_index: 232000\n",
      "current_index: 233000\n",
      "current_index: 234000\n",
      "current_index: 235000\n",
      "current_index: 236000\n",
      "current_index: 237000\n",
      "current_index: 238000\n",
      "current_index: 239000\n",
      "current_index: 240000\n",
      "current_index: 241000\n",
      "current_index: 242000\n",
      "current_index: 243000\n",
      "current_index: 244000\n",
      "current_index: 245000\n",
      "current_index: 246000\n",
      "current_index: 247000\n",
      "current_index: 248000\n",
      "current_index: 249000\n",
      "current_index: 250000\n",
      "current_index: 251000\n",
      "current_index: 252000\n",
      "current_index: 253000\n",
      "current_index: 254000\n",
      "current_index: 255000\n",
      "current_index: 256000\n",
      "current_index: 257000\n",
      "current_index: 258000\n",
      "current_index: 259000\n",
      "current_index: 260000\n",
      "current_index: 261000\n",
      "current_index: 262000\n",
      "current_index: 263000\n",
      "current_index: 264000\n",
      "current_index: 265000\n",
      "current_index: 266000\n",
      "current_index: 267000\n",
      "current_index: 268000\n",
      "current_index: 269000\n",
      "current_index: 270000\n",
      "current_index: 271000\n",
      "current_index: 272000\n",
      "current_index: 273000\n",
      "current_index: 274000\n",
      "current_index: 275000\n",
      "current_index: 276000\n",
      "current_index: 277000\n",
      "current_index: 278000\n",
      "current_index: 279000\n",
      "current_index: 280000\n",
      "current_index: 281000\n",
      "current_index: 282000\n",
      "current_index: 283000\n",
      "current_index: 284000\n",
      "current_index: 285000\n",
      "current_index: 286000\n",
      "current_index: 287000\n",
      "current_index: 288000\n",
      "current_index: 289000\n",
      "current_index: 290000\n",
      "current_index: 291000\n",
      "current_index: 292000\n",
      "current_index: 293000\n",
      "current_index: 294000\n",
      "current_index: 295000\n",
      "current_index: 296000\n",
      "current_index: 297000\n",
      "current_index: 298000\n",
      "current_index: 299000\n",
      "current_index: 300000\n",
      "current_index: 301000\n",
      "current_index: 302000\n",
      "current_index: 303000\n",
      "current_index: 304000\n",
      "current_index: 305000\n",
      "current_index: 306000\n",
      "current_index: 307000\n",
      "current_index: 308000\n",
      "current_index: 309000\n",
      "current_index: 310000\n",
      "current_index: 311000\n",
      "current_index: 312000\n",
      "current_index: 313000\n",
      "current_index: 314000\n",
      "current_index: 315000\n",
      "current_index: 316000\n",
      "current_index: 317000\n",
      "current_index: 318000\n",
      "current_index: 319000\n",
      "current_index: 320000\n",
      "current_index: 321000\n",
      "current_index: 322000\n",
      "current_index: 323000\n",
      "current_index: 324000\n",
      "current_index: 325000\n",
      "current_index: 326000\n",
      "current_index: 327000\n",
      "current_index: 328000\n",
      "current_index: 329000\n",
      "current_index: 330000\n",
      "current_index: 331000\n",
      "current_index: 332000\n",
      "current_index: 333000\n",
      "current_index: 334000\n",
      "current_index: 335000\n",
      "current_index: 336000\n",
      "current_index: 337000\n",
      "current_index: 338000\n",
      "current_index: 339000\n",
      "current_index: 340000\n",
      "current_index: 341000\n",
      "current_index: 342000\n",
      "current_index: 343000\n",
      "current_index: 344000\n",
      "current_index: 345000\n",
      "current_index: 346000\n",
      "current_index: 347000\n",
      "current_index: 348000\n",
      "current_index: 349000\n",
      "current_index: 350000\n",
      "current_index: 351000\n",
      "current_index: 352000\n",
      "current_index: 353000\n",
      "current_index: 354000\n",
      "current_index: 355000\n",
      "current_index: 356000\n",
      "current_index: 357000\n",
      "current_index: 358000\n",
      "current_index: 359000\n",
      "current_index: 360000\n",
      "current_index: 361000\n",
      "current_index: 362000\n",
      "current_index: 363000\n",
      "current_index: 364000\n",
      "current_index: 365000\n",
      "current_index: 366000\n",
      "current_index: 367000\n",
      "current_index: 368000\n",
      "current_index: 369000\n",
      "current_index: 370000\n",
      "current_index: 371000\n",
      "current_index: 372000\n",
      "current_index: 373000\n",
      "current_index: 374000\n",
      "current_index: 375000\n",
      "current_index: 376000\n",
      "current_index: 377000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current_index: 378000\n",
      "current_index: 379000\n",
      "current_index: 380000\n",
      "current_index: 381000\n",
      "current_index: 382000\n",
      "current_index: 383000\n",
      "current_index: 384000\n",
      "current_index: 385000\n",
      "current_index: 386000\n",
      "current_index: 387000\n",
      "current_index: 388000\n",
      "current_index: 389000\n",
      "current_index: 390000\n",
      "current_index: 391000\n",
      "current_index: 392000\n",
      "current_index: 393000\n",
      "current_index: 394000\n",
      "current_index: 395000\n",
      "current_index: 396000\n",
      "current_index: 397000\n",
      "current_index: 398000\n",
      "current_index: 399000\n",
      "current_index: 400000\n",
      "current_index: 401000\n",
      "current_index: 402000\n",
      "current_index: 403000\n",
      "current_index: 404000\n",
      "current_index: 405000\n",
      "current_index: 406000\n",
      "current_index: 407000\n",
      "current_index: 408000\n",
      "current_index: 409000\n",
      "current_index: 410000\n",
      "current_index: 411000\n",
      "current_index: 412000\n",
      "current_index: 413000\n",
      "current_index: 414000\n",
      "current_index: 415000\n",
      "current_index: 416000\n",
      "current_index: 417000\n",
      "current_index: 418000\n",
      "current_index: 419000\n",
      "current_index: 420000\n",
      "current_index: 421000\n",
      "current_index: 422000\n",
      "current_index: 423000\n",
      "current_index: 424000\n",
      "current_index: 425000\n",
      "current_index: 426000\n",
      "current_index: 427000\n",
      "current_index: 428000\n",
      "current_index: 429000\n",
      "current_index: 430000\n",
      "current_index: 431000\n",
      "current_index: 432000\n",
      "current_index: 433000\n",
      "current_index: 434000\n",
      "current_index: 435000\n",
      "current_index: 436000\n",
      "current_index: 437000\n",
      "current_index: 438000\n",
      "current_index: 439000\n",
      "current_index: 440000\n",
      "current_index: 441000\n",
      "current_index: 442000\n",
      "current_index: 443000\n",
      "current_index: 444000\n",
      "current_index: 445000\n",
      "current_index: 446000\n",
      "current_index: 447000\n",
      "current_index: 448000\n",
      "current_index: 449000\n",
      "current_index: 450000\n",
      "current_index: 451000\n",
      "current_index: 452000\n",
      "current_index: 453000\n",
      "current_index: 454000\n",
      "current_index: 455000\n",
      "current_index: 456000\n",
      "current_index: 457000\n",
      "current_index: 458000\n",
      "current_index: 459000\n",
      "current_index: 460000\n",
      "current_index: 461000\n",
      "current_index: 462000\n",
      "current_index: 463000\n",
      "current_index: 464000\n",
      "current_index: 465000\n",
      "current_index: 466000\n",
      "current_index: 467000\n",
      "current_index: 468000\n",
      "current_index: 469000\n",
      "current_index: 470000\n",
      "current_index: 471000\n",
      "current_index: 472000\n",
      "current_index: 473000\n",
      "current_index: 474000\n",
      "current_index: 475000\n",
      "current_index: 476000\n",
      "current_index: 477000\n",
      "current_index: 478000\n",
      "current_index: 479000\n",
      "current_index: 480000\n",
      "current_index: 481000\n",
      "current_index: 482000\n",
      "current_index: 483000\n",
      "current_index: 484000\n",
      "current_index: 485000\n",
      "current_index: 486000\n",
      "current_index: 487000\n",
      "current_index: 488000\n",
      "current_index: 489000\n",
      "current_index: 490000\n",
      "current_index: 491000\n",
      "current_index: 492000\n",
      "current_index: 493000\n",
      "current_index: 494000\n",
      "current_index: 495000\n",
      "current_index: 496000\n",
      "current_index: 497000\n",
      "current_index: 498000\n",
      "current_index: 499000\n",
      "current_index: 500000\n",
      "current_index: 501000\n",
      "current_index: 502000\n",
      "current_index: 503000\n",
      "current_index: 504000\n",
      "current_index: 505000\n",
      "current_index: 506000\n",
      "current_index: 507000\n",
      "current_index: 508000\n",
      "current_index: 509000\n",
      "current_index: 510000\n",
      "current_index: 511000\n",
      "current_index: 512000\n",
      "current_index: 513000\n",
      "current_index: 514000\n",
      "current_index: 515000\n",
      "current_index: 516000\n",
      "current_index: 517000\n",
      "current_index: 518000\n",
      "current_index: 519000\n",
      "current_index: 520000\n",
      "current_index: 521000\n",
      "current_index: 522000\n",
      "current_index: 523000\n",
      "current_index: 524000\n",
      "current_index: 525000\n",
      "current_index: 526000\n",
      "current_index: 527000\n",
      "current_index: 528000\n",
      "current_index: 529000\n",
      "current_index: 530000\n",
      "current_index: 531000\n",
      "current_index: 532000\n",
      "current_index: 533000\n",
      "current_index: 534000\n",
      "current_index: 535000\n",
      "current_index: 536000\n",
      "current_index: 537000\n",
      "current_index: 538000\n",
      "current_index: 539000\n",
      "current_index: 540000\n",
      "current_index: 541000\n",
      "(541635, 13)\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "common_table_columns = ['md5', 'feature_num', 'feature_list']\n",
    "common_table = pd.read_table('./ctr_cvr_data/BuyWeight_common_features_skeleton_test_sample_2_percent.csv', \n",
    "                                sep=',', header=None, names=common_table_columns, engine = 'python')\n",
    "feature_name_list = ['UserID', 'User_CateIDs', 'User_ShopIDs', 'User_BrandIDs', 'User_NodeIDs', 'User_Cluster', \n",
    "                     'User_ClusterID', 'User_Gender', 'User_Age', 'User_Level1', 'User_Level2', \n",
    "                     'User_Occupation', 'User_Geo']\n",
    "field_id_name = {'101':'UserID',\n",
    "                 '109_14':'User_CateIDs',\n",
    "                 '110_14':'User_ShopIDs',\n",
    "                 '127_14':'User_BrandIDs',\n",
    "                 '150_14':'User_NodeIDs',\n",
    "                 '121':'User_Cluster',\n",
    "                 '122':'User_ClusterID',\n",
    "                 '124':'User_Gender',\n",
    "                 '125':'User_Age',\n",
    "                 '126':'User_Level1',\n",
    "                 '127':'User_Level2',\n",
    "                 '128':'User_Occupation',\n",
    "                 '129':'User_Geo'}\n",
    "# 为了减少内存占用，方便单机版运行，先去掉Multi-hot特征\n",
    "#black_list = set(['109_14','110_14','127_14','150_14'])\n",
    "black_list = set(['110_14','150_14'])\n",
    "entire_fea_dict = {}\n",
    "for k,v in field_id_name.items():\n",
    "    entire_fea_dict[v] = []\n",
    "for index, row in common_table.iterrows():\n",
    "    feature_arr = row['feature_list'].split('\\001')\n",
    "    fea_dict = {}\n",
    "    for k,v in field_id_name.items():\n",
    "        fea_dict[k] = []\n",
    "    for fea_kv in feature_arr:\n",
    "        fea_field_id = fea_kv.split('\\002')[0]\n",
    "        fea_id_val = fea_kv.split('\\002')[1]\n",
    "        fea_id = fea_id_val.split('\\003')[0]\n",
    "        fea_val = fea_id_val.split('\\003')[1]\n",
    "        #print(fea_field_id,fea_id,fea_val)\n",
    "        if fea_field_id in black_list:\n",
    "            continue\n",
    "        # Multi-Hot IDs类特征保留前100个ID\n",
    "        if len(fea_dict[fea_field_id]) < 100:\n",
    "            fea_dict[fea_field_id].append(fea_id)\n",
    "    #print(fea_dict)\n",
    "    for k,v in fea_dict.items():\n",
    "        if len(v) == 0:\n",
    "            entire_fea_dict[field_id_name[k]].append('<PAD>')\n",
    "        else:\n",
    "            entire_fea_dict[field_id_name[k]].append('|'.join(v))\n",
    "    if index % 1000 == 0:\n",
    "       print(\"current_index:\",index)\n",
    "#print(entire_fea_dict)    \n",
    "\n",
    "entire_fea_table = pd.DataFrame(data=entire_fea_dict, columns=feature_name_list)\n",
    "print(entire_fea_table.shape)\n",
    "common_table = common_table.drop('feature_list',axis=1)\n",
    "common_table = pd.concat([common_table, entire_fea_table], axis=1, join_axes=[common_table.index])\n",
    "\n",
    "common_table.to_csv('./ctr_cvr_data/BuyWeight_sampled_common_features_skeleton_test_sample_feature_column.csv',index=False)\n",
    "print(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>md5</th>\n",
       "      <th>feature_num</th>\n",
       "      <th>UserID</th>\n",
       "      <th>User_CateIDs</th>\n",
       "      <th>User_ShopIDs</th>\n",
       "      <th>User_BrandIDs</th>\n",
       "      <th>User_NodeIDs</th>\n",
       "      <th>User_Cluster</th>\n",
       "      <th>User_ClusterID</th>\n",
       "      <th>User_Gender</th>\n",
       "      <th>User_Age</th>\n",
       "      <th>User_Level1</th>\n",
       "      <th>User_Level2</th>\n",
       "      <th>User_Occupation</th>\n",
       "      <th>User_Geo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0010d0b9633bb5b0</td>\n",
       "      <td>250</td>\n",
       "      <td>66015</td>\n",
       "      <td>455028|451998|451100|445269|445990|450099|4557...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3520924|3505215|3588720|3541711|3801132|382945...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3438670</td>\n",
       "      <td>3438756</td>\n",
       "      <td>3438769</td>\n",
       "      <td>3438771</td>\n",
       "      <td>3438777</td>\n",
       "      <td>3438782</td>\n",
       "      <td>3864886</td>\n",
       "      <td>3864889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0012aad1f55312b6</td>\n",
       "      <td>170</td>\n",
       "      <td>121803</td>\n",
       "      <td>451822|451095|449537|449301|455342|449077|4490...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3518975|3697970|3784310|3821497|3698768|345218...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3438658</td>\n",
       "      <td>3438766</td>\n",
       "      <td>3438768</td>\n",
       "      <td>3438772</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3438782</td>\n",
       "      <td>3864885</td>\n",
       "      <td>3864889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0013e5c24e8dd3a6</td>\n",
       "      <td>617</td>\n",
       "      <td>135732</td>\n",
       "      <td>452511|450721|449079|450276|450656|449078|4493...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3707605|3632935|3809314|3703188|3700287|356905...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3438670</td>\n",
       "      <td>3438756</td>\n",
       "      <td>3438769</td>\n",
       "      <td>3438771</td>\n",
       "      <td>3438777</td>\n",
       "      <td>3438782</td>\n",
       "      <td>3864885</td>\n",
       "      <td>3864889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>001459b610a7c186</td>\n",
       "      <td>395</td>\n",
       "      <td>235356</td>\n",
       "      <td>450880|456589|450870|450658|451641|451639|4508...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3765068|3620179|3619511|3668302|3610026|378643...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001efa0ef1001dd1</td>\n",
       "      <td>849</td>\n",
       "      <td>131668</td>\n",
       "      <td>445600|450229|449070|449317|450658|449178|4509...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3628671|3853135|3805119|3848839|3651862|372880...</td>\n",
       "      <td>&lt;PAD&gt;</td>\n",
       "      <td>3438685</td>\n",
       "      <td>3438762</td>\n",
       "      <td>3438769</td>\n",
       "      <td>3438774</td>\n",
       "      <td>3438778</td>\n",
       "      <td>3438782</td>\n",
       "      <td>3864885</td>\n",
       "      <td>3864888</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                md5  feature_num  UserID  \\\n",
       "0  0010d0b9633bb5b0          250   66015   \n",
       "1  0012aad1f55312b6          170  121803   \n",
       "2  0013e5c24e8dd3a6          617  135732   \n",
       "3  001459b610a7c186          395  235356   \n",
       "4  001efa0ef1001dd1          849  131668   \n",
       "\n",
       "                                        User_CateIDs User_ShopIDs  \\\n",
       "0  455028|451998|451100|445269|445990|450099|4557...        <PAD>   \n",
       "1  451822|451095|449537|449301|455342|449077|4490...        <PAD>   \n",
       "2  452511|450721|449079|450276|450656|449078|4493...        <PAD>   \n",
       "3  450880|456589|450870|450658|451641|451639|4508...        <PAD>   \n",
       "4  445600|450229|449070|449317|450658|449178|4509...        <PAD>   \n",
       "\n",
       "                                       User_BrandIDs User_NodeIDs  \\\n",
       "0  3520924|3505215|3588720|3541711|3801132|382945...        <PAD>   \n",
       "1  3518975|3697970|3784310|3821497|3698768|345218...        <PAD>   \n",
       "2  3707605|3632935|3809314|3703188|3700287|356905...        <PAD>   \n",
       "3  3765068|3620179|3619511|3668302|3610026|378643...        <PAD>   \n",
       "4  3628671|3853135|3805119|3848839|3651862|372880...        <PAD>   \n",
       "\n",
       "  User_Cluster User_ClusterID User_Gender User_Age User_Level1 User_Level2  \\\n",
       "0      3438670        3438756     3438769  3438771     3438777     3438782   \n",
       "1      3438658        3438766     3438768  3438772       <PAD>     3438782   \n",
       "2      3438670        3438756     3438769  3438771     3438777     3438782   \n",
       "3        <PAD>          <PAD>       <PAD>    <PAD>       <PAD>       <PAD>   \n",
       "4      3438685        3438762     3438769  3438774     3438778     3438782   \n",
       "\n",
       "  User_Occupation User_Geo  \n",
       "0         3864886  3864889  \n",
       "1         3864885  3864889  \n",
       "2         3864885  3864889  \n",
       "3           <PAD>    <PAD>  \n",
       "4         3864885  3864888  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "common_table.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.array([1,2])\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
